{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8a6f1eec-a30d-4e43-b51a-bdd99d71887a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "34dff443-a1e7-4aac-94e0-0fefda71458a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bcebb625-40d5-4345-8045-5d547a4cb0fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(\"the-verdict.txt\"):\n",
    "    url = (\n",
    "        \"https://raw.githubusercontent.com/rasbt/\"\n",
    "        \"LLMs-from-scratch/main/ch02/01_main-chapter-code/\"\n",
    "        \"the-verdict.txt\"\n",
    "    )\n",
    "    file_path = \"the-verdict.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e502f7dc-f15a-49b7-9486-c1adcdf59966",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(url,timeout=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0426794a-9973-4aa6-95b5-399a8832d93e",
   "metadata": {},
   "outputs": [],
   "source": [
    "response.raise_for_status()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ecac730a-b8f6-4d4b-b816-5b79ebf4a769",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(file_path,\"wb\") as f:\n",
    "    f.write(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3b37888a-a4a2-4706-9195-e04014b9ac2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"the-verdict.txt\",\"r\",encoding='utf-8') as f:\n",
    "    raw_text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7c6558ee-a94e-4ff1-b66f-4ac475316a64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of character: 20479\n"
     ]
    }
   ],
   "source": [
    "print(\"Total number of character:\",len(raw_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aa572c35-759c-4af6-8981-c26938d2281e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I HAD always thought Jack Gisburn rather a cheap genius--though a good fellow enough--so it was no \n"
     ]
    }
   ],
   "source": [
    "print(raw_text[:99])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6f1dabc3-c427-474a-8d8e-a9d07e047197",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "30a4510d-8d80-45ae-b490-82b38a0a0889",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"Hello, world. This, is a test.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b521d67e-4f3f-44af-91f6-4eaff3aa2f79",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = re.split(r'(\\s)',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "470bc8a6-0953-44a3-aee8-3468d6d12652",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello,', ' ', 'world.', ' ', 'This,', ' ', 'is', ' ', 'a', ' ', 'test.']\n"
     ]
    }
   ],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6cba9051-c322-4027-a4c7-664afa3e3799",
   "metadata": {},
   "outputs": [],
   "source": [
    "result1 = re.split(r'([,.]|\\s)',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "62725002-59da-4369-a30e-cd489aec2dec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', '', ' ', 'world', '.', '', ' ', 'This', ',', '', ' ', 'is', ' ', 'a', ' ', 'test', '.', '']\n"
     ]
    }
   ],
   "source": [
    "print(result1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "73cc4795-5153-4eb6-941c-5dd9e7a4f4fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "result2 = [item for item in result1 if item.strip()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "413c496b-6ca5-4fdc-8470-d51c2f59ae02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', 'world', '.', 'This', ',', 'is', 'a', 'test', '.']\n"
     ]
    }
   ],
   "source": [
    "print(result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3e24f488-07e5-4d3f-8141-434f6e563915",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"Hello, world. Is this-- a test?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "df260203-cdc8-45a7-9d9a-d470cb7a190f",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = re.split(r'([,.:;?_!\"()\\']|--|\\s)', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "62651041-e64c-4c84-9a67-9c2706d325c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = [item.strip() for item in result if item.strip()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f4301126-bb63-42c2-be40-fc11f365288a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', 'world', '.', 'Is', 'this', '--', 'a', 'test', '?']\n"
     ]
    }
   ],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0e261e1c-83b0-49d1-99e7-f2f1f7b4c52d",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed = re.split(r'([,.:;?_!\"()\\']|--|\\s)', raw_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d44b0aa4-4a44-42e1-b9af-1d5760f9a1c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed = [item.strip() for item in preprocessed if item.strip()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cf39a9fd-428b-4256-a213-c9f31c8d07cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', 'HAD', 'always', 'thought', 'Jack', 'Gisburn', 'rather', 'a', 'cheap', 'genius', '--', 'though', 'a', 'good', 'fellow', 'enough', '--', 'so', 'it', 'was', 'no', 'great', 'surprise', 'to', 'me', 'to', 'hear', 'that', ',', 'in']\n"
     ]
    }
   ],
   "source": [
    "print(preprocessed[:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e5cd399a-48df-43f3-bc19-27e2ddc67954",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4690\n"
     ]
    }
   ],
   "source": [
    "print(len(preprocessed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "db54ea51-7f69-4881-9dba-1c082214a654",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = sorted(set(preprocessed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1a3be3df-91b4-4ddb-8776-3f2544a1b599",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(all_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "abcedfa8-227c-4bb6-967b-0f4f9056d9d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1130\n"
     ]
    }
   ],
   "source": [
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "46677bb8-50cb-4be4-830f-b91009fb4e0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = {token:integer for integer,token in enumerate(all_words)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ad6d7fe4-35f9-41a2-90d1-1bc583ddfe48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('!', 0)\n",
      "('\"', 1)\n",
      "(\"'\", 2)\n",
      "('(', 3)\n",
      "(')', 4)\n",
      "(',', 5)\n",
      "('--', 6)\n",
      "('.', 7)\n",
      "(':', 8)\n",
      "(';', 9)\n",
      "('?', 10)\n",
      "('A', 11)\n",
      "('Ah', 12)\n",
      "('Among', 13)\n",
      "('And', 14)\n",
      "('Are', 15)\n",
      "('Arrt', 16)\n",
      "('As', 17)\n",
      "('At', 18)\n",
      "('Be', 19)\n",
      "('Begin', 20)\n",
      "('Burlington', 21)\n",
      "('But', 22)\n",
      "('By', 23)\n",
      "('Carlo', 24)\n",
      "('Chicago', 25)\n",
      "('Claude', 26)\n",
      "('Come', 27)\n",
      "('Croft', 28)\n",
      "('Destroyed', 29)\n",
      "('Devonshire', 30)\n",
      "('Don', 31)\n",
      "('Dubarry', 32)\n",
      "('Emperors', 33)\n",
      "('Florence', 34)\n",
      "('For', 35)\n",
      "('Gallery', 36)\n",
      "('Gideon', 37)\n",
      "('Gisburn', 38)\n",
      "('Gisburns', 39)\n",
      "('Grafton', 40)\n",
      "('Greek', 41)\n",
      "('Grindle', 42)\n",
      "('Grindles', 43)\n",
      "('HAD', 44)\n",
      "('Had', 45)\n",
      "('Hang', 46)\n",
      "('Has', 47)\n",
      "('He', 48)\n",
      "('Her', 49)\n",
      "('Hermia', 50)\n"
     ]
    }
   ],
   "source": [
    "for i,item in enumerate(vocab.items()):\n",
    "    print(item)\n",
    "    if i >=50:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e7fbe7d9-1c8b-41ee-92b1-3e49ecef882d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleTokenizerV1:\n",
    "    def __init__(self,vocab):\n",
    "        self.str_to_int = vocab\n",
    "        self.int_to_str = {i:s for s,i in vocab.items()}\n",
    "\n",
    "    def encode(self, text):\n",
    "        preprocessed = re.split(r'([,.:;?_!\"()\\']|--|\\s)', text)\n",
    "\n",
    "        preprocessed = [\n",
    "            item.strip() for item in preprocessed if item.strip()\n",
    "        ]\n",
    "        ids = [self.str_to_int[s] for s in preprocessed]\n",
    "        return ids\n",
    "\n",
    "    def decode(self, ids):\n",
    "        text = \" \".join([self.int_to_str[i] for i in ids])\n",
    "        ## replace space before the specified punctuations\n",
    "        text = re.sub(r'\\s+([,.?!\"()\\'])', r'\\1',text)\n",
    "        return text\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e5e6de3c-3b49-415f-9849-ba3ea2e25c1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = SimpleTokenizerV1(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b2d1d928-a84f-49d4-8f23-a39d6f76fd43",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\"\"\"It's the last he painted, you know,\" \n",
    "           Mrs. Gisburn said with pardonable pride.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e4e2a801-3a99-4142-a65b-5c058a16f1f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = tokenizer.encode(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "6521a46b-8385-4072-b90a-5423109c20dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 56, 2, 850, 988, 602, 533, 746, 5, 1126, 596, 5, 1, 67, 7, 38, 851, 1108, 754, 793, 7]\n"
     ]
    }
   ],
   "source": [
    "print(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "39dcbf3d-9a24-4ebd-b781-5ebc231336b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\" It\\' s the last he painted, you know,\" Mrs. Gisburn said with pardonable pride.'"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "24110fea-6233-4015-81dd-6631983fe4a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\" It\\' s the last he painted, you know,\" Mrs. Gisburn said with pardonable pride.'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(tokenizer.encode(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cda01fb0-41c1-4000-9d71-8122c9363a68",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "eeb86806-73d7-4f9f-b93b-cdf815062dfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = SimpleTokenizerV1(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "fc1c7430-b7eb-411a-8593-da3ac2fd6211",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"Hello, do you like tea. Is this-- a test?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "8376d5d2-22fc-417e-abba-48b9b962e605",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Hello'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[47], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[38], line 12\u001b[0m, in \u001b[0;36mSimpleTokenizerV1.encode\u001b[1;34m(self, text)\u001b[0m\n\u001b[0;32m      7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.:;?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[0;32m      9\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m     10\u001b[0m     item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()\n\u001b[0;32m     11\u001b[0m ]\n\u001b[1;32m---> 12\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n",
      "\u001b[1;31mKeyError\u001b[0m: 'Hello'"
     ]
    }
   ],
   "source": [
    "tokenizer.encode(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "efd4105a-656e-42f6-aa07-90086c2bac74",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_tokens = sorted(list(set(preprocessed)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "09f2101b-bdb8-470f-a053-d6a18e4e1260",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_tokens.extend([\"<|endoftext|>\",\"<|unk|>\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "6ee2970d-deee-44a1-9449-6bf62f26e509",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = {token:integer for integer,token in enumerate(all_tokens)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "dfe37344-c27c-425f-aa18-896e45412a16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1132"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "64ba260a-ec00-462b-8573-153c10015a51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ('younger', 1127)\n",
      "1 ('your', 1128)\n",
      "2 ('yourself', 1129)\n",
      "3 ('<|endoftext|>', 1130)\n",
      "4 ('<|unk|>', 1131)\n"
     ]
    }
   ],
   "source": [
    "for i,item in enumerate(list(vocab.items())[-5:]):\n",
    "    print(i,item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "d5a7f8e0-7db8-40da-a544-741375f4e7c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleTokenizerV2:\n",
    "    def __init__(self,vocab):\n",
    "        self.str_to_int = vocab\n",
    "        self.int_to_str = {i:s for s,i in vocab.items()}\n",
    "\n",
    "    def encode(self, text):\n",
    "        preprocessed = re.split(r'([,.:;?_!\"()\\']|--|\\s)',text)\n",
    "        preprocessed = [item.strip() for item in preprocessed if item.strip()]\n",
    "        preprocessed = [\n",
    "            item if item in self.str_to_int\n",
    "            else \"<|unk|>\" for item in preprocessed\n",
    "        ]\n",
    "\n",
    "        ids = [self.str_to_int[s] for s in preprocessed]\n",
    "        return ids\n",
    "\n",
    "    def decode(self, ids):\n",
    "        text=\" \".join([self.int_to_str[i] for i in ids])\n",
    "        text = re.sub(r'\\s+([,.:;?!\"()\\'])', r'\\1',text)\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9c7671a5-6ce5-48ef-9f97-cf4ee3c92ea9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = SimpleTokenizerV2(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "11f53713-cff3-4ccd-bf04-ff9a2ae6e926",
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = \"Hello, do you like tea?\"\n",
    "text2 = \"In the sunlit terraces of the palace.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "0606dd96-990a-4773-ba36-59b8a0de715f",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \" <|endoftext|> \".join((text1, text2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "82671c9e-ce13-4851-b2dc-a0bdb53581fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello, do you like tea? <|endoftext|> In the sunlit terraces of the palace.\n"
     ]
    }
   ],
   "source": [
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "dfcce5ce-0bbb-4a84-aee7-ce00726a0c56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1131, 5, 355, 1126, 628, 975, 10, 1130, 55, 988, 956, 984, 722, 988, 1131, 7]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f861a369-4468-44b2-8cb8-fd89d9394239",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<|unk|>, do you like tea? <|endoftext|> In the sunlit terraces of the <|unk|>.'"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(tokenizer.encode(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48c82288-2794-4422-9ca5-d88f7441b57e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
